Preprocessing QC statistics

Noam, July 2023

In [1]:
%load_ext autoreload
%autoreload 2
In [ ]:
import os
MOMAPS_HOME = '/home/labs/hornsteinlab/Collaboration/MOmaps_Noam/MOmaps'
MOMAPS_DATA_HOME = '/home/labs/hornsteinlab/Collaboration/MOmaps'
LOGS_PATH = os.path.join(MOMAPS_HOME, 'outputs', 'preprocessing', 'spd','logs','no_downsample')
PLOT_PATH = os.path.join(MOMAPS_HOME, 'src', 'preprocessing', 'notebooks','figures','Neurons')
os.chdir(MOMAPS_HOME)
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
sns.set(style="whitegrid", font_scale=1.5)
sns.color_palette("husl", 8)
plt.rcParams["image.cmap"] = "Set1"
from tqdm.notebook import tqdm
from src.common.lib.preprocessing_utils import rescale_intensity
from src.common.lib.images_qc import *
#sys.path.insert(1, "/home/labs/hornsteinlab/Collaboration/MOmaps_Sagy/MOmaps/src/common/lib")
import contextlib
import io
import matplotlib
import warnings
warnings.filterwarnings('ignore', category=pd.core.common.SettingWithCopyWarning)
from src.common.lib.qc_config_tmp import *
from src.common.lib.image_sampling_utils import *
In [ ]:
df = log_files_qc(LOGS_PATH)
In [ ]:
# choose batches
#batches = [f'batch{i}' for i in range (3,7)] + ['batch7_16bit','batch8_16bit','batch9_16bit']
batches=[f'batch{i}_16bit_no_downsample' for i in range(3,10)]
batches

Raw Files

In [5]:
root_directory_raw = os.path.join(MOMAPS_DATA_HOME, 'input', 'images', 'raw', 'SpinningDisk')

batches_raw = [batch.replace("_16bit_no_downsample","") for batch in batches]
raws = run_validate_folder_structure(root_directory_raw, False, panels, markers,PLOT_PATH,marker_info,
                                    cell_lines_to_cond, reps, cell_lines_for_disp, expected_dapi_raw,
                                     batches=batches_raw)
batch3
Folder structure is invalid. Missing paths:
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/raw/SpinningDisk/batch3/FUSHomozygous/panelA
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/raw/SpinningDisk/batch3/FUSHomozygous/panelB
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/raw/SpinningDisk/batch3/FUSHomozygous/panelC
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/raw/SpinningDisk/batch3/TDP43/panelA
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/raw/SpinningDisk/batch3/TDP43/panelB
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/raw/SpinningDisk/batch3/TDP43/panelC
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/raw/SpinningDisk/batch3/TBK1/panelA
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/raw/SpinningDisk/batch3/TBK1/panelB
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/raw/SpinningDisk/batch3/TBK1/panelC
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/raw/SpinningDisk/batch3/WT/panelA
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/raw/SpinningDisk/batch3/WT/panelB
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/raw/SpinningDisk/batch3/WT/panelC
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/raw/SpinningDisk/batch3/SCNA/panelA
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/raw/SpinningDisk/batch3/SCNA/panelB
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/raw/SpinningDisk/batch3/SCNA/panelC
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/raw/SpinningDisk/batch3/FUSRevertant/panelA
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/raw/SpinningDisk/batch3/FUSRevertant/panelB
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/raw/SpinningDisk/batch3/FUSRevertant/panelC
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/raw/SpinningDisk/batch3/OPTN/panelA
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/raw/SpinningDisk/batch3/OPTN/panelB
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/raw/SpinningDisk/batch3/OPTN/panelC
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/raw/SpinningDisk/batch3/FUSHeterozygous/panelA
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/raw/SpinningDisk/batch3/FUSHeterozygous/panelB
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/raw/SpinningDisk/batch3/FUSHeterozygous/panelC
Some files are bad:
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_output
CellProfiler_output
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
CellProfiler_DAPI-count
Total Sites:  43325
========
batch4
Folder structure is valid.
All files exists.
Total Sites:  64800
========
batch5
Folder structure is valid.
All files exists.
Total Sites:  64800
========
batch6
Folder structure is valid.
All files exists.
Total Sites:  64800
========
batch7
Folder structure is valid.
All files exists.
Total Sites:  64800
========
batch8
Folder structure is valid.
All files exists.
Total Sites:  64800
========
batch9
Folder structure is valid.
All files exists.
Total Sites:  64800
========
====================

Processed

In [6]:
root_directory_proc = os.path.join(MOMAPS_DATA_HOME, 'input', 'images', 'processed', 'spd2',
                              'SpinningDisk')
procs = run_validate_folder_structure(root_directory_proc, True, panels, markers,PLOT_PATH,marker_info,
                                    cell_lines_to_cond, reps, cell_lines_for_disp, expected_dapi_raw,
                                     batches=batches)
batch3_16bit_no_downsample
Folder structure is invalid. Missing paths:
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/FUSHomozygous/Untreated/G3BP1
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/FUSHomozygous/Untreated/NONO
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/FUSHomozygous/Untreated/SQSTM1
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/FUSHomozygous/Untreated/KIF5A
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/FUSHomozygous/Untreated/TDP43
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/FUSHomozygous/Untreated/FMRP
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/FUSHomozygous/Untreated/PURA
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/FUSHomozygous/Untreated/CD41
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/FUSHomozygous/Untreated/Phalloidin
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/TDP43/Untreated/G3BP1
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/TDP43/Untreated/NONO
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/TDP43/Untreated/SQSTM1
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/TDP43/Untreated/KIF5A
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/TDP43/Untreated/TDP43
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/TDP43/Untreated/FMRP
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/TDP43/Untreated/PURA
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/TDP43/Untreated/CD41
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/TDP43/Untreated/Phalloidin
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/TBK1/Untreated/G3BP1
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/TBK1/Untreated/NONO
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/TBK1/Untreated/SQSTM1
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/TBK1/Untreated/KIF5A
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/TBK1/Untreated/TDP43
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/TBK1/Untreated/FMRP
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/TBK1/Untreated/PURA
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/TBK1/Untreated/CD41
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/TBK1/Untreated/Phalloidin
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/WT/Untreated/G3BP1
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/WT/Untreated/NONO
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/WT/Untreated/SQSTM1
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/WT/Untreated/KIF5A
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/WT/Untreated/TDP43
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/WT/Untreated/FMRP
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/WT/Untreated/PURA
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/WT/Untreated/CD41
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/WT/Untreated/Phalloidin
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/WT/stress/G3BP1
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/WT/stress/NONO
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/WT/stress/SQSTM1
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/WT/stress/KIF5A
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/WT/stress/TDP43
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/WT/stress/FMRP
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/WT/stress/PURA
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/WT/stress/CD41
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/WT/stress/Phalloidin
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/SCNA/Untreated/G3BP1
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/SCNA/Untreated/NONO
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/SCNA/Untreated/SQSTM1
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/SCNA/Untreated/KIF5A
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/SCNA/Untreated/TDP43
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/SCNA/Untreated/FMRP
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/SCNA/Untreated/PURA
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/SCNA/Untreated/CD41
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/SCNA/Untreated/Phalloidin
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/FUSRevertant/Untreated/G3BP1
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/FUSRevertant/Untreated/NONO
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/FUSRevertant/Untreated/SQSTM1
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/FUSRevertant/Untreated/KIF5A
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/FUSRevertant/Untreated/TDP43
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/FUSRevertant/Untreated/FMRP
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/FUSRevertant/Untreated/PURA
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/FUSRevertant/Untreated/CD41
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/FUSRevertant/Untreated/Phalloidin
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/OPTN/Untreated/G3BP1
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/OPTN/Untreated/NONO
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/OPTN/Untreated/SQSTM1
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/OPTN/Untreated/KIF5A
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/OPTN/Untreated/TDP43
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/OPTN/Untreated/FMRP
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/OPTN/Untreated/PURA
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/OPTN/Untreated/CD41
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/OPTN/Untreated/Phalloidin
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/FUSHeterozygous/Untreated/G3BP1
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/FUSHeterozygous/Untreated/NONO
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/FUSHeterozygous/Untreated/SQSTM1
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/FUSHeterozygous/Untreated/KIF5A
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/FUSHeterozygous/Untreated/TDP43
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/FUSHeterozygous/Untreated/FMRP
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/FUSHeterozygous/Untreated/PURA
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/FUSHeterozygous/Untreated/CD41
/home/labs/hornsteinlab/Collaboration/MOmaps/input/images/processed/spd2/SpinningDisk/batch3_16bit_no_downsample/FUSHeterozygous/Untreated/Phalloidin
All files exists.
Total Sites:  36891
========
batch4_16bit_no_downsample
Folder structure is valid.
All files exists.
Total Sites:  51984
========
batch5_16bit_no_downsample
Folder structure is valid.
All files exists.
Total Sites:  52702
========
batch6_16bit_no_downsample
Folder structure is valid.
All files exists.
Total Sites:  64056
========
batch7_16bit_no_downsample
Folder structure is valid.
All files exists.
Total Sites:  63906
========
batch8_16bit_no_downsample
Folder structure is valid.
All files exists.
Total Sites:  63923
========
batch9_16bit_no_downsample
Folder structure is valid.
All files exists.
Total Sites:  63453
========
====================

Difference between Raw and Processed

In [7]:
display_diff(batches, raws, procs, PLOT_PATH)
batch3_16bit_no_downsample
========
batch4_16bit_no_downsample
========
batch5_16bit_no_downsample
========
batch6_16bit_no_downsample
========
batch7_16bit_no_downsample
========
batch8_16bit_no_downsample
========
batch9_16bit_no_downsample
========
In [9]:
#for batch in list(range(3,9)) + ['7_16bit','8_16bit','9_16bit']:  
for batch in batches:
    with contextlib.redirect_stdout(io.StringIO()):
        var = sample_and_calc_variance(root_directory_proc, batch, 
                                       sample_size_per_markers=200, num_markers=26)
    print(f'{batch} var: ',var)
batch3_16bit_no_downsample var:  0.014236584165425802
batch4_16bit_no_downsample var:  0.014105815782507117
batch5_16bit_no_downsample var:  0.014413725611422278
batch6_16bit_no_downsample var:  0.016610036695837587
batch7_16bit_no_downsample var:  0.016083577771303287
batch8_16bit_no_downsample var:  0.01649681842883865
batch9_16bit_no_downsample var:  0.01657061673241471

Number of sites in each batch and cell line

In [5]:
df_no_per = df[df.batch!='Perturbations']
plot_sites_count(df_no_per, expected_raw, lines_order, custom_palette, split_to_reps=True)

Number of Cells in Site for each batch and cell line

In [20]:
df_no_empty_sites = df_no_per[df_no_per.n_valid_tiles !=0]
plot_cell_count(df_no_empty_sites, lines_order, custom_palette, whole_cells=True)

plot_cell_count(df_no_empty_sites, lines_order, custom_palette, whole_cells=False)
# can add norm=True to norm by max

number of valid tiles per image (site)

In [22]:
plot_n_valid_tiles_count(df_no_per, custom_palette,reps, batch_min=3, batch_max=9)
In [23]:
plot_p_valid_tiles_count(df_no_per, custom_palette,reps, batch_min=3, batch_max=9)

Heatmap QC per batch, panel and cell line(tiles that passed QC condition)

In [25]:
plot_hm(df_no_per, split_by='rep', rows='cell_line', columns='panel')

Assessing Staining Reproducibility and Outliers

In [ ]:
for batch in batches:
    print(batch)
#     if 'batch7' in batch:
#         continue
    #batch_num = batch.replace('batch',"")
    run_calc_hist_new(batch,cell_lines_for_disp, markers, ncols=7, nrows=4,
                           hist_sample=10,sample_size_per_markers=200)
    print("="*30)
batch3_16bit_no_downsample
/home/labs/hornsteinlab/Collaboration/MOmaps_Noam/MOmaps/src/common/lib/images_qc.py:831: UserWarning: This figure includes Axes that are not compatible with tight_layout, so results might be incorrect.
  plt.tight_layout()
/home/labs/hornsteinlab/Collaboration/MOmaps/anaconda3/momaps_37/lib/python3.7/site-packages/IPython/core/pylabtools.py:151: UserWarning: This figure includes Axes that are not compatible with tight_layout, so results might be incorrect.
  fig.canvas.print_figure(bytes_io, **kw)
==============================
batch4_16bit_no_downsample
==============================
batch5_16bit_no_downsample
==============================
batch6_16bit_no_downsample
==============================
batch7_16bit_no_downsample
==============================
batch8_16bit_no_downsample
==============================
batch9_16bit_no_downsample
In [ ]:
# save notebook as HTML ( the HTML will be saved in the same folder the original script is)
from IPython.display import display, Javascript
display(Javascript('IPython.notebook.save_checkpoint();'))
os.system('jupyter nbconvert --to html src/preprocessing/notebooks/no_ds/cell_count_stats_analysis_no_downsample.ipynb')
In [ ]: